import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import App;
import util.HTTPUtil;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * <p>description: </p>
 *
 * @author chenrui
 * @since 2018-11-26
 */
@Service
public class SpiderApp {


    public static String mainPageUrl = "https://sj.qq.com/myapp/category.htm?orgame=1";
    public static String jsonUrl = "https://sj.qq.com/myapp/cate/appList.htm?orgame=1&categoryId={categoryId}&pageSize=20&pageContext={pageContext}";
    private static ConcurrentLinkedQueue<String> categoryIdList = new ConcurrentLinkedQueue<>();

    //线程池
	//线程池讲解在 https://blog.csdn.net/Hello_Ray/article/details/84306356 博客中有详细介绍
    private static ExecutorService executorService = Executors.newFixedThreadPool(20);


    public void run() {
        String content = HTTPUtil.sendGet(mainPageUrl).getContent();
        Document document = Jsoup.parse(content);
        Elements elements = document.getElementsByClass("menu-junior");
        Elements tags = elements.get(0).getElementsByTag("li");
        for(int i=0; i<tags.size(); i++) {
            String htmlId = tags.get(i).attr("id");
            if(!StringUtils.isEmpty(htmlId)) {
                String categoryId = htmlId.replace("cate-", "");
                categoryIdList.add(categoryId);
            }
        }

//        sendJsonUrl(jsonUrl, "-10");
        //开启线程池去处理数据
        while(!CollectionUtils.isEmpty(categoryIdList)) {
            final String categoryId = categoryIdList.poll();
            executorService.submit(new Runnable() {
                @Override
                public void run() {
                    sendJsonUrl(jsonUrl, categoryId);
                }
            });
        }
        executorService.shutdown();
    }


    /**
     * 发送json请求
     */
    public void sendJsonUrl(String url, String categoryId) {
        url = url.replace("{categoryId}", categoryId).replace("{pageContext}", "0");
		//发送http请求，解析json串
        String content = HTTPUtil.sendGet(url).getContent();
        JSONObject jsonObject = JSON.parseObject(content);
        String objString = jsonObject.getString("obj");

		//每一次请求，是分页进行的。所以还有进行分页处理，每次pageSize增加，直至取到空数据为止。
        int index = 0;
        do {
            JSONArray jsonArray = jsonObject.getJSONArray("obj");
            for(int i=0; i<jsonArray.size(); i++) {
                App app = jsonArray.getJSONObject(i).toJavaObject(App.class);
                //-------获得app的数据，对app数据进行操作。----//
				System.out.println(i+"  "+app.getAppName() +"   "+url);
				//-----------------------------------------//

            }
            //每执行一次pageContext加上20
            int pageContext = (++index)*20;
            String againUrl = jsonUrl;
            againUrl = againUrl.replace("{categoryId}", categoryId).replace("{pageContext}", pageContext+"");
            content = HTTPUtil.sendGet(againUrl).getContent();
            jsonObject = JSON.parseObject(content);
            objString = jsonObject.getString("obj");

        } while (!StringUtils.isEmpty(objString));

    }

}
